#!/usr/bin/env python
# -*- coding:utf-8 -*-
import re
from lxml import etree
from dosql import *
URL_FIX = re.compile("#+|/+") 

class HtmlParserTool():
    def __init__(self, data):
        self.html_tree = self.bulidEtree(data)
    
    def bulidEtree(self, data):
        html_tree = None
        try:
            html_tree = etree.HTML(data.lower().decode('utf-8', 'ignore'))
        except Exception, e:
            print "build html_tree", e
        return html_tree
        
    def extractUrlsFromData(self, prefix_url):
        urls = []
        if self.html_tree == None:
            return urls
        try:
            hrefs = self.html_tree.xpath('//a')
            for href in hrefs:
                url = href.get('href')
                url = self.fixURL(url, prefix_url)
                if url != None:
                    urls.append(url)
        except Exception, e:
            print 'extractUrlsFromData', e
        return urls
    
    def extractItems(self, pipeline = None, extra = None):
        try:
            return pipeline(self.html_tree, extra)
        except Exception, e:
            return None
        pass
    
    def cusFixURL(self, url, prefix):
        return self.fixURL(url, prefix)
    
    def fixURL(self, url, prefix):
        if url == None:return None
        idx = url.find('http')
        if idx == -1:
            from urlparse import urlparse
            prefixparser = urlparse(prefix)
            url = prefixparser.scheme + '://' + prefixparser.hostname +'/' + url
        elif idx != 0:
            return None
        try:
            protocol, url = url.split('://')
            url, i = URL_FIX.subn(r'/', url)
            url, i = URL_FIX.subn(r'/' ,url)
            url = protocol +'://' + url
            if url[-1] == '/':
                url = url[0:-1] 
        except Exception, e:
            print "fixURL",e
            return None
        return url
    
    
        
if __name__ == '__main__':
    prefix = "http://www.zhihu.com/question/20109291"
    from urlparse import urlparse
    prefix = urlparse(prefix)
    print prefix.hostname
    